# -*- coding: utf-8 -*-
from decimal import Decimal

import pandas as pd
import scrapy

from apps.listed_company.listed_company.items import NetNewRmbLoans


class NewRmbLoansSpider(scrapy.Spider):
    listed_exchange = '同花顺-新增人民币贷款'
    name = 'new_rmb_loans'
    headers = {
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
        "Accept-Language": "zh,zh-TW;q=0.9,en-US;q=0.8,en;q=0.7,zh-CN;q=0.6",
        "Cache-Control": "max-age=0",
        "Connection": "keep-alive",
        "Referer": "https://data.10jqka.com.cn/macro/cpi/",
        "Sec-Fetch-Dest": "document",
        "Sec-Fetch-Mode": "navigate",
        "Sec-Fetch-Site": "same-origin",
        "Sec-Fetch-User": "?1",
        "Upgrade-Insecure-Requests": "1",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
        "sec-ch-ua": "\"Chromium\";v=\"130\", \"Google Chrome\";v=\"130\", \"Not?A_Brand\";v=\"99\"",
        "sec-ch-ua-mobile": "?0",
        "sec-ch-ua-platform": "\"Windows\""
    }
    custom_settings = {
        "DUPEFILTER_DEBUG": True,
        'DUPEFILTER_CLASS': 'scrapy.dupefilters.BaseDupeFilter'
    }

    def start_requests(self):
        url = "https://data.10jqka.com.cn/macro/loan/"
        yield scrapy.Request(url, callback=self.parse_list, headers=self.headers)

    def parse_list(self, response, **kwargs):
        df = pd.read_html(response.text)[0]
        lines = df.to_dict('records')
        for data in lines:
            item = NetNewRmbLoans(**{
                'time': data[('月份', '月份')].replace('-', ''),
                'new_rmb_loans_total': data[('新增人民币贷款', '总额(亿元)')],
                'new_rmb_loans_yoy': Decimal(data[('新增人民币贷款', '同比')].strip('%')) / 100,
                'new_rmb_loans_mom': Decimal(data[('新增人民币贷款', '环比')].strip('%')) / 100,
                'accumulated_rmb_loans_total': data[('累计人民币贷款', '总额(亿元)')],
                'accumulated_rmb_loans_yoy': Decimal(data[('累计人民币贷款', '同比')].strip('%')) / 100,
            })
            # print(item)
            yield item


if __name__ == "__main__":
    from scrapy import cmdline

    cmdline.execute("scrapy crawl new_rmb_loans".split())
